
from pyquery import  PyQuery as pq


#header = [{'User-Agent': 'Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/46.0.2490.76 Mobile Safari/537.36'},{'User-Agent':'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-us) AppleWebKit/534.50 (KHTML, like Gecko) Version/5.1 Safari/534.50'},{'User-Agent':'Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0)'},{'User-Agent':'Mozilla/5.0 (Windows NT 6.1; rv:2.0.1) Gecko/20100101 Firefox/4.0.1'},{'User-Agent': 'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; 360SE)'}]
def get_url(url):
    req = pq(url,encoding="gbk")
    list_url = req('.col-md-3 a')
    list_url_=[]
    for i in list_url.items():
        list_url_.append(url+i.attr('href'))
    return (list_url_)

def get_txt(list_url_):
    for url in list_url_:
        req = pq(url,encoding="gbk")
        # req1 = pq(url,encoding="gbk")
        text = req('.panel-body ').text()
        # text1 = req1('.panel-body ').text()

        a = "一秒记住【笔趣阁 www.biqukan.cc】，更新快，无弹窗，免费读！"
        b = "本章未完，点击下一页继续阅读"
        title_pq =req('.readTitle').text()+'\n'
        text=text.replace(a,'')
        text = text.replace(b,'')
        text = text.replace('\ufffd', '')
        # text1=text1.replace(b,'')
        # print(text)
        text =title_pq + text
        with open("novle1.txt","a+") as f:
            f.write(text)
    print("大功告成")
url = get_url("http://www.biqukan.cc/book/20461/")
get_txt(url)

